# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy_redis.spiders import RedisCrawlSpider
from FrSpider.items import FrxxzItem

# import sys
# reload(sys)
# sys.setdefaultencoding('utf8')


# class YbduSpider(CrawlSpider):
class YbduSpider(RedisCrawlSpider):
    name = 'biquge'
    # allowed_domains = ['biquge.info']
    # start_urls = ['https://www.biquge.info/10_10240/']

    # redis_key
    redis_key = "biquge:start_urls"

    rules = (
        Rule(LinkExtractor(allow=r'\d{7}.html'), callback='parse_item', follow=True),
    )

    # 动态获取allowed_domains
    def __init__(self, *args, **kwargs):
        domain = kwargs.pop("domain","")
        self.allowed_domains = filter(None, domain.split(","))
        super(YbduSpider, self).__init__(*args, **kwargs)


    def parse_item(self, response):
        # response = unicode(response, "utf-8")#.encode("utf8")
        item = FrxxzItem()
        title = response.xpath("//div[@class='bookname']/h1/text()").extract()
        content = response.xpath("//div[@id='content']").extract()
        item['title'] = title[0]
        item['content'] = content[0]
        yield item